#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Jul 23 21:48:49 2018
@project: 天池比赛-A股主板上市公司公告信息抽取
@group: MZH_314
@author: LHQ
"""
import os
import re

import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import KFold, cross_val_score
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import chi2
from sklearn.pipeline import Pipeline
from sklearn.externals import joblib
from sklearn import svm

from reportIE.utils import seg
from reportIE.feature import Feature


def process_doc(doc):
    """分词过滤

    Args
    ----
    doc : str
        待分词的字符串文本

    Returns
    -------
    new_doc : str
        用空格拼接分词过滤的词组
        主要是方便作为sklearn的CountVectorizer输入参数
    """
    words = [w for w in seg(doc) if len(w) > 1 and not re.search("^\d+$", w)]
    new_doc = " ".join(words)
    return new_doc


class SentenceFeature:
    """句子特征的构造

    主要有两类特征组成：模式特征和词语特征
    1.模式特征是指句子有是否含有某个特定的模式，数值格式（均通过正则表达式获取）
    2.词语特征是词袋模型特征，这里采用0-1二值特征

    Attributes
    ----------
    word_feat : 含有transform方法的对象
        transform方法会将docs转换成0-1值的特征矩阵，目前用的是sklearn里的类，即组合了
        Select和Countvecterizer的pipeline,会对词语先做特征选择再构造0-1特征矩阵

    pattern_feat : 含有build_feat_matrix方法的对象
        这里的pattern_feat来自自己定义的Featur类, 该类通过add_pattern方法增加模式,
        通过build_feat_matrix构造是否含有事先定义的模式的0-1特征矩阵
    """

    def __init__(self, word_feat):
        self.word_feat = word_feat

        self.pattern_feat = Feature()
        self.pattern_feat.add_pattern("\d+-\d+-\d+")
        self.pattern_feat.add_pattern("\d+股")
        self.pattern_feat.add_pattern("后")
        self.pattern_feat.add_pattern("占")
        self.pattern_feat.add_pattern("%")
        self.pattern_feat.add_pattern("[于至]")
        self.pattern_feat.add_pattern("\d+\.\d+")
  
    def build_pattern_feature(self, docs):
        """构造句子中含有特定模式结构的特征
        模式结构由初始化中的Feature类的add_pattern方法
        添加
        
        Args
        ----
        docs : iterable
            每次迭代得到一条句子

        Returns
        -------
        pattern_feature : 0-1特征矩阵
        """
        pattern_feature = self.pattern_feat.build_feat_matrix(docs)
        return pattern_feature
    
    def build_words_feature(self, docs):
        """构造词语特征
        
        Args
        ----
        docs : iterable
            每次迭代得到一条句子

        Returns
        -------
        words_feature : 特征矩阵
        """
        docs_new = [process_doc(doc) for doc in docs]
        words_feature = self.word_feat.transform(docs_new).toarray()
        return words_feature
    
    def build_feature(self, docs):
        """构造完整的特征， 合并了模式特征和词语特征

        Args
        ----
        docs : iterable
            每次迭代得到一条句子

        Returns
        -------
        feature : 特征矩阵
        """
        pattern_feature = self.build_pattern_feature(docs)
        words_feature = self.build_words_feature(docs)
        feature = np.concatenate((pattern_feature, words_feature), axis=1)
        return feature
    
    @classmethod
    def from_modelfile(cls, feat_path=None):
        if feat_path is None:
            feat_path = "models/zengjianchi/text_sentence_feat.m"
        word_feat = joblib.load(feat_path)
        feat = cls(word_feat)
        return feat


if __name__ == "__main__":
    trainingdata_path = os.path.abspath("../data/training_data/zengjianchi_text.csv")

    df = pd.read_csv(trainingdata_path)

    label = df['is_target'].tolist()
    docs = df['origin_text'].tolist()

    """训练词语特征构造器及特征选择器"""
    # 分词过滤
    docs_new = [process_doc(doc) for doc in docs]
    # 训练0-1值特征转换器
    vectorizer = CountVectorizer(binary=True)
    vectorizer.fit_transform(docs_new)
    # 特征选择
    features_tmp = vectorizer.transform(docs_new)
    select = SelectKBest(chi2, k=300)
    select.fit_transform(features_tmp, label)
    # 用pipeline连接特征转换器与特征选择器
    feat_pipe = Pipeline(steps=[('vectorizer', vectorizer), ("select", select)])
   
    """构造特征"""
    sentence_feat = SentenceFeature(feat_pipe)
    feature = sentence_feat.build_feature(docs)

    """分类器训练"""
    # 支持向量机分类
    clf = svm.SVC()

    # 交叉验证看效果
    k_fold = KFold(n_splits=5)
    scores = cross_val_score(clf, feature, label, cv=k_fold, n_jobs=-1)
    print(scores)    

    # 训练
    clf.fit(feature, label)

    """ 模型保存"""
    joblib.dump(clf, "models/zengjianchi/text_sentence_clf.m")
    joblib.dump(feat_pipe, "models/zengjianchi/text_sentence_feat.m")
    
